#!/bin/bash

#
# GuangZhou - doc file, need download!
#

city='成都市'
base_url="http://www.cdstats.chengdu.gov.cn"
list_url=$base_url"/htm/list_020110.html"
abs_path=n
doc_file=n
pdf_file=y

year=$1

if [ -z $year ];then
	echo "usage: $0 <year>"
	echo "example: $0 2021"
	exit 1
fi

file_name="${year}年${city}国民经济和社会发展统计公报"

if [ "$abs_path" == "y" ];then
	year_uri=`curl -s -L $list_url | grep -E "${year}年${city}国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
else
	#year_uri=`curl -s -L $list_url | grep -E "${year}年.*国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g' | sed 's/\.//'`
	#year_uri=`curl -s -L $list_url | grep -E "${year}年${city}国民经济和社会发展统计公报|${year}年.*经济和社会发展概况" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	year_uri=`curl -s -L $list_url | grep -o -P "<a href=\"/htm/detail.*?title=\"${year}年${city}国民经济和社会发展统计公报\">" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
fi

#echo "$year uri: $year_uri"
#exit 0

if [ -z "$year_uri" ];then
	echo "err: parse year uri failed!"
	exit 2
fi

if [ ${year_uri:0:4} == "http" ];then
	year_url=${year_uri}
else
	year_url="${base_url}"${year_uri}
fi

echo "$year URL: $year_url"

report="`curl -s -L $year_url`"
curl -s -L $year_url -o tmp.html

if [ "$doc_file" == "y" ];then
	#doc_url=`echo $report | grep -E "${year}年.*国民经济和社会发展统计公报.doc" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	doc_url=`cat tmp.html | grep -E "${year}年.*国民经济和社会发展统计公报.doc" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	echo "Doc Url: $doc_url"
	curl -sL $doc_url -o "${year}年${city}国民经济和社会发展统计公报.doc"
	exit 0
fi

if [ "$pdf_file" == "y" ];then
	#pdf_uri=`echo $report | grep -E "${year}年.*国民经济和社会发展统计公报.pdf" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	pdf_uri=`cat tmp.html | grep -E "${year}年.*国民经济和社会发展统计公报.pdf" | awk -F href= '{ print $2 }' | awk '{ print $1 }'|sed 's/"//g'`
	echo "PDF Uri: $pdf_uri"

	# url encode
	#pdf_uri=`echo $pdf_uri | xxd -ps | tr -d '\n' | sed -r 's/(..)/%\1/g'`
	pdf_uri=`echo -n $pdf_uri | python3 -c "import sys; from urllib.parse import quote; print(quote(sys.stdin.read()));"`

	if [ ${pdf_uri:0:4} == "http" ];then
		pdf_url=${pdf_uri}
	else
		pdf_url="${base_url}"${pdf_uri}
	fi

	curl -sL $pdf_url -o "${file_name}.pdf"

	pdftotext ${file_name}.pdf
	if [ $? -ne 0 ];then
		echo "exec pdftotext ${file_name}.pdf failed!"
		exit 0
	fi
fi

if [ -f ${file_name}.txt ];then
	sed -i ':a;N;s/\n//g;ta' ${file_name}.txt
	report="`cat ${file_name}.txt`"
else
	exit 0
fi

# dict map
declare -A fields #(["常住人口"]=0, ["生产总值"]=0, ["一般公共预算收入"]=0, ["一般公共预算支出"]=0)

# for sequence
#key_list=("常住人口" "地区生产总值" "一般公共预算收入" "一般公共预算支出")
key_list=("年末常住人口" "实现地区生产总值" "一般公共预算收入" "一般公共预算支出")

#for key in ${!fields[*]};
for key in ${key_list[*]};
do
	fields[$key]=`echo $report | awk -F "$key" '{ print $2 }' | awk -F '，' '{ print $1 }' | perl -pe 's/<.*?>//g' | sed 's/\[.*\]//g'`
	#fields[$key]=`cat tmp.html | awk -F "$key" '{ print $2 }' | awk -F '，' '{ print $1 }' | perl -pe 's/<.*?>//g' | sed 's/\[.*\]//g'`
	echo "$key: ${fields[$key]}"
done

exit 0

